import scrapy
from w3lib.html import remove_tags

class QuotesSpider(scrapy.Spider):

    name = "bbc"
    start_urls = ['http://www.bbc.com','https://www.solidot.org']


    def start_requests(self):

        for url in self.start_urls:
            yield scrapy.Request(url=url,
                                 callback=self.parse)

    def parse(self, response):

        div = response.css('div.module__content')
        url = div.css('a.block-link__overlay-link::attr(href)').extract()
        for a in url:
            yield response.follow(url=a, callback=self.parse_bbc,meta={'url':a})

    def parse_bbc(self, response):
        # def extract_with_css(query):
        #     return response.css(query).extract_first().strip()

        url = response.meta['url']
        if response.css('h1.story-body__h1::text').extract()==[]:
            return 1
        yield {
            'url':url,
            'title': response.css('h1.story-body__h1::text').extract(),
            'content': response.css('div.story-body__inner').extract(),
            'dateline':  response.css('div.date  ::text').extract_first(),
        }